#!/usr/local/bin/perl -w

# search on this form:  "inurl:301412 site:worldbank.org"

# Open csv
# Do a search on google.com for each
# Print report

use WWW::Mechanize;
use Text::CSV_XS;

# Need to stop if getting google errors -- first will try backing off 5 minutes
$google_error_count = 0;

$script_dir = "C:/Documents and Settings/wb253181/workspace2/InGoogleTest/";

$inventory = $script_dir . "DH List.csv";
if ( open( INVENTORY, "<" . $inventory ) ) { }
else { print "FAILURE in openning $inventory for reading\n"; exit(); }

$results = $script_dir . "results.csv";
if ( open( RESULTS, ">" . $results ) ) { }
else { print "FAILURE in openning $results for reading\n"; exit(); }

$agent = WWW::Mechanize->new();

# Eat the first three lines since it's just header information
$i = 0;
while ( $i < 3 ) {
	my $inventory_line = <INVENTORY>;
	$i++;
}

my $inventory_csv = Text::CSV_XS->new( { binary => 1, eol => $/ } );

while ( defined( $inventory_line = <INVENTORY> ) ) {
	$matches = "N/A";
	$googledotcom_status = "INTERNAL ERROR";

	if ( $inventory_csv->parse($inventory_line) ) {
		@values = $inventory_csv->fields;
	}
	else {
		my $err = $inventory_csv->error_input;
		print "parse () failed on argument: ", $err, "\n";
		exit(0);
	}

	$site_home_page_url = $values[2];
	$sitePK       = $values[1];
	$original_status = $values[4];

	if ($site_home_page_url =~ /.*(intranet|intnewsletter).*/ or $site_home_page_url eq "") {	
		$is_external_url = 0;
	} else {
		$is_external_url = 1;
	}
	
	# FIRST GET FROM GOOGLE.COM
	$google_query = "inurl:$sitePK site:worldbank.org";
	print "\n";
	print "google query \= $google_query\n";
	print "site_home_page_url \= $site_home_page_url\n";

	$google_url = "http://www.google.com/search?q=$google_query";
	# $google_url = "http://extsearch.worldbank.org/servlet/SiteSearchServlet?q=$google_query";

	$agent->get($google_url);
	if (!$is_external_url) {
		print "intranet url -- not testing\n";
		$googledotcom_status = "INTRANET URL -- not tested";
	} elsif ( $agent->status eq 200 ) {
		$google_response = $agent->{content};
		# <font size=-1>Results <b>1</b> - <b>10</b> of about <b>192</b> from <b>worldbank.org</b> for <b>inurl:239419</b>
		if ( $google_response =~
/.+of\s+[about]*\s*\<b\>([0-9,]*)\<\/b\> from.+/
		  )
		{
			$matches = $1;
			print "Google.com Match for $sitePK $matches\n";
			$googledotcom_status = "LIVE";
		}
		else {
			print "Google.com No Match for $sitePK\n";
			$googledotcom_status = "NOT LIVE";
		}
	}
	else {
		$google_error_count++;
		$server_status = $agent->status;
		print "received error $server_status from web server\n";
		print "bad url: $google_url\n";
		$googledotcom_status = "SERVER ERROR -- number $google_error_count from Google.com";
		if ($google_error_count eq 3) {
			$error_message = "Already received $google_error_count errors from Google.com, so stopping execution\n";
			print RESULTS $error_message;
			die($error_message);
		}
		$error_message = "***** PAUSING FIFTEEN MINUTES -- received error from Google.com *****\n";
		print $error_message;
		print RESULTS $error_message;
		sleep(60*15);
	}
	
	
	# NOW GET FROM EXTERNAL APPLIANCE
	
	$appliance_first_time = 1;
	$appliance_repeat = 0;
	
	while ($appliance_first_time or $appliance_repeat) {
		$appliance_first_time = 0;
		$google_appliance_query = "inurl:$sitePK site:worldbank.org";
		print "$google_appliance_query\n";
		$appliance_status = "INTERNAL ERROR";


		if ($is_external_url){
			$google_appliance_url = "http://extsearch.worldbank.org/servlet/SiteSearchServlet?q=$google_appliance_query";		
		} else {
			# http://intranetnca.worldbank.org/servlet/main?hNavID=1&menuPK=64169128&pagePK=64231990&piPK=888663&srcType=1&theSitePK=33705&query=hobbs&selectItem=All%20Results
			$google_appliance_url = "http://intranetnca.worldbank.org/servlet/main?hNavID=1&menuPK=64169128&pagePK=64231990&piPK=888663&srcType=1&theSitePK=33705&query=$google_appliance_query&selectItem=All%20Results";
		}

		$agent->get($google_appliance_url);

		if ( $agent->status eq 200 ) {
			$google_appliance_response = $agent->{content};
			# <font size=-1>Results <b>1</b> - <b>10</b> of about <b>192</b> from <b>worldbank.org</b> for <b>inurl:239419</b>
			if ( ($is_external_url and $google_appliance_response =~ /.+Your search did not match any documents.+/)
				or 
				 (!$is_external_url and $google_appliance_response =~ /.+We are sorry\, there are no results that match your search for.+/) )
			{
				print "Appliance No Match for $sitePK\n";
				$appliance_status = "NOT LIVE";
				if ($appliance_repeat) {$appliance_repeat = 0;} else {$appliance_repeat = 1; sleep(2);}
			} else {
				if (!$appliance_repeat) {
					print "Appliance Match for $sitePK\n";
					$appliance_status = "LIVE";
				} else {
					$appliance_repeat = 0;
					print "Appliance Match for $sitePK (after repeat)\n";
					$appliance_status = "LIVE (after repeat)";				
				}
			}
		} else {
			$server_status = $agent->status;
			if (!$appliance_repeat) {
				print "received error $server_status from web server\n";
				print "bad url: $google_url\n";
				$appliance_status = "SERVER ERROR";
			} else {
				$appliance_repeat = 0;
				print "received error $server_status from web server (after repeat)\n";
				print "bad url: $google_url\n";
				$appliance_status = "SERVER ERROR (after repeat)";				
			}
			if ($appliance_repeat) {$appliance_repeat = 0;} else {$appliance_repeat = 1; sleep(2);};
		}
	}
	
	# NOW PRINT OUT THE LINE OF INFORMATION
	chomp $inventory_line;
	
	# Figure out whether to print Internal or External
	if ($is_external_url) {$ext_or_int = "External";} else {$ext_or_int = "Internal \(or no url in inventory\)";};
	
	# Figure out whether a change is being suggested
	if ($googledotcom_status eq "LIVE" or $appliance_status eq "LIVE" or $appliance_status eq "LIVE \(after repeat\)") {
		$overall_status = "Live";
	} else {
		$overall_status = "Production";
	}
	
	$is_equiv_prod{"Migrating"} = 1;
	$is_equiv_prod{"Support"} = 1;
	$is_equiv_prod{"Deleted"} = 1;
	$is_equiv_prod{"Production"} = 1;
	
	#  For the purposes of this script, Production == Migrating == Support == Deleted
	#  Can't really check Remove 
	#  Live has to equal Live
	if (($original_status eq $overall_status) 
		or ($overall_status eq "Production" and ($is_equiv_prod{$original_status}))) {
		$change_status = "Leave unchanged";
	} elsif ($overall_status eq "Production" and $original_status eq "Live") {
		$change_status = "Change to Production";
	} elsif ($overall_status eq "Live" and $is_equiv_prod{$original_status}) {
		$change_status = "Chage to Live";
	} elsif ($original_status eq "Remove") {
		$change_status = "Not checking Removes";
	} else 	{
		$change_status = "**INTERNAL ERROR**";
	}
	
	# Really should use the CVS module to print this out
	print RESULTS "$inventory_line\,$googledotcom_status,\"$matches\"\,$appliance_status\,$ext_or_int\,$change_status\n";
	sleep(30);
}

